# -*- coding:utf-8 -*-
# 开发时间: 2019/10/21 13:29
import csv
import os
import requests
import json
import time

# 类型
# category=movie
# 电视剧
# category=%E7%94%B5%E8%A7%86%E5%89%A7
# 综艺
# %E7%BB%BC%E8%89%BA
# 八卦
# category=gossip



url = 'https://www.toutiao.com/api/pc/feed/?category=movie&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1654DCABE1B925&cp=5DAEFB69F2655E1&_signature=awDdiwAANrFiTKeeh8-DrmsA3Z'


    #
    # url = 'https://www.toutiao.com/api/pc/feed/?category=movie&utm_source=toutiao&widen=1&max_behot_time=1571671034&tadrequire=true&as=A1951D8A4D74E63&cp=5DAD241E26B30E1&_signature=lmjXnwAAy9GfJK2KkQgjoJZo14'

headers = {
'cookie': 'tt_webid=6749789110511093262; csrftoken=ac40800e2a735d1d85e0519cea42abce; _ga=GA1.2.518825367.1571558150; _gid=GA1.2.899471419.1571558150; tt_webid=6749789110511093262; s_v_web_id=5b89a54ecf7ad267ec0af993b86736ab; WEATHER_CITY=%E5%8C%97%E4%BA%AC; UM_distinctid=16ded1c138f2b0-0806f225708001-e343166-144000-16ded1c139075f; CNZZDATA1259612802=1077707740-1571636962-https%253A%252F%252Fwww.toutiao.com%252F%7C1571636962; __tasessionId=klye7i85x1571661940641',

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'

}



# j解析数据
def pares_first(url ,headers,num):

    item ={}
    count = 1
    response = requests.get(url=url, headers=headers)
    print(response.text)
    result = response.content.decode('utf8')
    resultdata = json.loads(result)
    print(len(resultdata['data']))

    print(resultdata['next']['max_behot_time'])
    next_time = resultdata['next']['max_behot_time']


    if next_time:

        for list in resultdata['data']:
            print('爬取第' + str(num) + '页' + '第' + str(count) + '项数据')
            try:
                # 标题
                print('标题      ' + list['title'])
                item['title'] = list['title']

                # 超链接 https://www.toutiao.com/a6750106121635103245/
                chao_lsit = 'https://www.toutiao.com/a' + list['item_id'] + '/'
                print('超链接      ' + chao_lsit)
                item['super_url'] = chao_lsit


                # 时间戳
                # print('时间戳'       + list['behot_time'])
                # item['time'] = list['behot_time']


                # 图片链接
                print('图片连接     ' + list['middle_image'])
                item['image_url'] = list['middle_image']


                name = item['title'].split('，')[0]
                print('图片名字     ' + name)

                image = down_image(item['image_url'])

                print('开始下载数据')

                # 下载图片
                filepath = write_image(name, image)

                item['image_path'] = filepath
                write_text(item)
                print('爬取第' + str(num) + '页' + '第' + str(count) + '项数据爬取完成')
                count += 1

            except:
                item['image_path'] = ''

                print('部分数据错误，已跳过')

        # 下一个请求的url地址

        url_2 = 'https://www.toutiao.com/api/pc/feed/?category=movie&utm_source=toutiao&widen=1&max_behot_time={}&tadrequire=true'.format(next_time)

        return url_2
    else:
        url_2 = 'https://www.toutiao.com/api/pc/feed/?category=movie&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1951D8A4D74E63&cp=5DAD241E26B30E1&_signature=lmjXnwAAy9GfJK2KkQgjoJZo14'
        print("===========重新刷新爬取数据==========")
        return url_2
#




# 下载图片的url
def down_image(ur):

    return requests.get(url=ur,headers=headers)


# 下载图片
def write_image(name, image):
    dirname = 'image'
    if not os.path.exists(dirname):
        os.mkdir(dirname)
    filepath = dirname + '/' + name + '.jpg'
    with open(filepath, 'wb') as fp:
        fp.write(image.content)
    return filepath



# 存入数据
def write_text(item):
    filename = 'jnri.csv'
    with open(filename, 'a', newline='', encoding='utf8') as fp:

            name2 = ['类型', '标题', '文章链接', '图片url', '图片位置',]
            writer = csv.DictWriter(fp, fieldnames=name2)

            if os.path.getsize(filename) == 0:
                 writer.writeheader()
            writer.writerow({'类型': 'move', '标题': item['title'], '文章链接': item['super_url'], '图片url': item['image_url'], '图片位置': item['image_path']})





if __name__ == '__main__':

    page = int(input("请输入需要爬取的页数"))

    for i in range(page):

        print('开始爬取第' + str(i + 1) + '页数据')
        try:
            url = pares_first(url, headers, i + 1)
            print('#' * 20)
            print(url)


        # except UnicodeEncodeError :
        #     print('第' + str(i + 1) + '页数据爬取失败,   转码错误')
        except KeyError:
            print('第' + str(i + 1) + '页数据爬取失败，   找不到目标键')

        time.sleep(2)
        print('第' + str(i + 1) + '页数据爬取完成')

# https://www.toutiao.com/api/pc/feed/?category=news_entertainment&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1C5DDBAEDBCD1D&cp=5DAD0CCD014D5E1&_signature=FzvytAAASpoed4ih7ZYn5hc78q
# # https://www.toutiao.com/api/pc/feed/?category=news_entertainment&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1B55D9A6DDCDA1&cp=5DAD2C2D2A215E1&_signature=FzvytAAASpoed4ih7Zb3Chc78q